package GetSummary;

use 5.012002;
use strict;
use warnings;

use File::Find;
use File::Basename;
use File::Spec;
use File::Path;

use Time::Local;
use Cwd;
use Fcntl;
use File::Copy::Recursive qw(fcopy rcopy dircopy fmove rmove dirmove);
use Scalar::Util qw(looks_like_number);
use Template;
use Time::SoFar qw( runtime runinterval figuretimes );
use Statistics::Descriptive;
use Statistics::R;

use List::MoreUtils qw{
  any all none notall true false
  firstidx first_index lastidx last_index
  insert_after insert_after_string
  apply indexes
  after after_incl before before_incl
  firstval first_value lastval last_value
  each_array each_arrayref
  pairwise natatime
  mesh zip uniq distinct minmax part
};

#require Exporter;

#our @ISA = qw(Exporter);

# Items to export into callers namespace by default. Note: do not export
# names by default without a very good reason. Use EXPORT_OK instead.
# Do not simply export all your public functions/methods/constants.

# This allows declaration	use GetSummary ':all';
# If you do not need this, moving things directly into @EXPORT or @EXPORT_OK
# will save memory.
#our %EXPORT_TAGS = ( 'all' => [ qw(

#) ] );

#our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } );

#our @EXPORT = qw(

#);
our $VERSION = '0.01';

# Need to be moved to sub later.
#my $ERRORCODE_INDEX;
#my $FSN_INDEX;
#my $ATEVER_INDEX;
#my $BOMREV_INDEX;
#my $DUT_FW_REV_INDEX;
#my $MODELNO_INDEX;
#my $DATE_INDEX;
#my $STNID_INDEX;

# Preloaded methods go here.

sub new {
	my $package = shift;
	return bless( {}, $package );

}

sub GetSummary {
	my $self = shift;

	my $workingdir       = shift;    # Current directory script is runing
	my $outputdir        = shift;    # HTML output file directory
	my $searchdirectory  = shift;    # The search directory
	my $stationtype      = shift;    # Station Type, RFCAL, BFT, etc
	my $TargetstationID  = shift;    # Station ID, REH_BFT_L_10.
	my $productID        = shift;    # Product ID, product ID in FSN.
	my $productBOM       = shift;    # BOM rev in FSN
	my $productFW        = shift;    # DUT FW version
	my $testcodeversion  = shift;    # Test software verion
	my $startdate        = shift;    # Start point of search range
	my $enddate          = shift;    # End poit of search ragne
	my $gen_dis_graphics = shift;    # Generate the distribution or not?
	my $outputfile       = shift;    # HTML file path.
	my $op_textRef       = shift;    # Tk text widget reference to log msg.

	logmsg( $op_textRef, "Starting summary ..." );

	print "parameters passed in: \n";
	print "	
		\$workingdir		= $workingdir,
		\$outputdir 		= $outputdir,
		\$searchdirectory	= $searchdirectory,
		\$stationtype		= $stationtype,
		\$TargetstationID	= $TargetstationID,
		\$productID			= $productID,
		\$productBOM		= $productBOM,
		\$productFW			= $productFW,
		\$testcodeversion	= $testcodeversion,
		\$startdate			= $startdate,
		\$enddate			= $enddate,
		\$gen_dis_graphics	= $gen_dis_graphics,
		\$outputfile 		= $outputfile  \n ";

	my %HoAA;    #This is a Hash of Array of reference to another Array
	my %AllLinesHoAA;
	my %ResultHoAA;    # has of the records with the time stamp in keys
	my %FirstHoA;
	my %SecondHoA;

	my %CommonFieldIndexH;
	my $CurrentLine;
	my @ResultTitles;
	my @ResultSubTitles;
	my @ResultUnits;
	my @ResultLowLimit;
	my @ResultHiLimit;

	my @AllFiles;

	my @LinesFromCurrFile;

	my $rindextmpOfFilePath;
	my @ResultLines;
	my @ResultLinesAoA;            # all matching records.
	my @FirstMatchingRecordAoA;    # The first matching record.
	my $CriteriaFieldSubTitle;
	my @TestList;
	my @ResultArrayUnique = ();
	my @FieldIndex;

	my %paretodataHoA;
	my @summarydataAoA;

	# Hash contains all error code of all matching records.
	my %ECOfAllMatchingRecordsH;
	my %FirstOccuranceH;
	my %SecondOccuranceH;
	my %PassedDUTQty;

	my @ATERevsUsed;
	my @BOMRevsUsed;
	my @DUTFWRevsUsed;
	my @STNsUsed;
	my @DUTPrefixUsed;
	my @ProdIDUsed;

	print "current work directory= $workingdir, \n\n";
	print "################## Start Of Script ######################### \n";

	################################################################
	# Define the search criteria here,                             #
	################################################################
	# which fields will be displayed.
	my @TestSearchList = (

		# The fields want to search in the title fields.
		# information
		'STN_ID',
		'FSN',

		#'PASS',
		'Start Time',    # this will tell the time and date tested.
		                 #'Start Time OS',
		'Error Code',
		'Modem Type',
		'BFT',           # this is the test time location
		'RFCAL',         # this is the test time location.
		'ATE Vers',

		#'Limit Vers',
		#'key',
		'BOM Rev',
		'FW Rev',
		"CMU_FW",
	);

	#my $noofcommonfields            = 11;
	my $decimallength               = 4;
	my $toperrorcodewillbedisplayed = 15;
	my $No_Of_Top_EC                = 10;
	my %searchlistH;
	for (@TestSearchList) { $searchlistH{$_} = 1; }

	##################################################################
	# General critiria
	##################################################################
	# which directory will be parsed
	my $SearchDir = $searchdirectory;

#if ( $debugmode == 1 ) {
#	$SearchDir ='\jasmine2\Misc\MfgTestData\Factory\rawdata\USB306\TDN-000953\RF\RF41\PartBUSB306\2009_18\data';
#}

	#print "\$SearchDir=$SearchDir \n";
	logmsg( $op_textRef, "\$SearchDir=$SearchDir \n" );

#if ( $SearchDir =~ /^\\/ ) { $SearchDir =~ s/\\/\/\//; }
#$SearchDir =~ s/^\\/\/\//;	# this is for raw code usage
# +: at lease match once. can't use '*', otherwise, it will match even nothing.just put '\' in front.
	$SearchDir =~ s/^\\+/\/\//;    # this is for userface usage

	#print "\$SearchDir=$SearchDir \n";
	$outputdir =~ s/^\\+/\/\//;

	# If display the matched record in seperate file.
	# 1 - The matching record in file will be displayed
	# 0 - The matching record in file will not be displayed
	#my $ShowSeperateFileRecord = 0;

	# find the matching subtitles.
	my $TargetSubTitle = '.*';

	# Just find the unique records? TRUE - Yes. FALSE - No.
	# my $UniqueResult = 'FALSE';

	my $ShowResults = 0;

	# omit the band info in error code. TRUE or FALSE;
	my $CombineErrorCode = 1;

	#will eliminated the time string from Start Time Field.
	my $KeepDateStampOnly = 0;

	my $gen_dis_graph = $gen_dis_graphics;

	##################################################################
	# Define critirias for Files
	##################################################################
 # Used to match the DUT type in file name, not that one in record.
 # The filename just uses the system type from conf file.
 # So there will be not accurate. So remove the match from filename.
 # Using the FSN to do the match. this one is used for part of chart output name
 # my $DUTType = 'MC8790';
 # used to match the STN, FLX_BFT_L_1 etc.
 # my $TargetSTNID = '';
 # matching the station ID, BFT, RFCAL, etc.
	my $TargetSTN   = $stationtype;        # can't be empty.
	my $TargetSTNID = $TargetstationID;    # can't be empty.

	# search data in between, 29/12/2006
	my $StartDate = $startdate;
	my $EndDate   = $enddate;

	##################################################################
	# Define critiria for matching records                           #
	##################################################################
	# this column is from test code itself. should be safe.
	#my $TargetATEVer = '.*';		#	'.' will match everything. $testcodeversion
	my $TargetATEVer = $testcodeversion;    #$testcodeversion

	# will match the BOM revision in FSN. should be safe
	my $TargetBOMRev = $productBOM;

# Don't use this for yield summary, this column won't be available before poweron.
	my $DUTFWRevision = $productFW;

	# Error Code in Records. safe.
	my $ErrorCode = '.*';

	# FSN prefix, it's factory ID.
	#my $FSNPrefix ='D681318009710';
	my $FSNPrefix = $productID;

	my $looptest;
	if   ( length($FSNPrefix) >= 13 ) { $looptest = 1; }
	else                              { $looptest = 0; }

	# Model No in record. such as 'MC8775', 'MC8775V', 'MC8755', etc.
	# can't match the model no here. if the DUT was not turned on,
	# there won't be any model no in the record. so all error code occured
	# before power on, will be ignored.
	# my $TargetModelInRecord = '';
	# Which FSNs will be subtracted in the result files.
	# The first digit of FSN is factory site.
	# X---Flex Shenzhen, S---San jose

	###################### define the Start and End point #########################
	my $unix_time_start;
	my $unix_time_end;
	if ( ( $StartDate !~ /^$/ ) && ( $EndDate !~ /^$/ ) ) {
		# my ( $mday_start, $mth_start, $year_start ) =
		my ( $year_start, $mth_start, $mday_start) =
		  # $StartDate =~ /(\d+)\/(\d+)\/(\d+)/;
		  $StartDate =~ /(\d+)\-(\d+)\-(\d+)/;
		# my ( $mday_end, $mth_end, $year_end ) =
		my ( $year_end, $mth_end, $mday_end) =
		  # $EndDate =~ /(\d+)\/(\d+)\/(\d+)/;
		  $EndDate =~ /(\d+)\-(\d+)\-(\d+)/;

		$unix_time_start =
		  scalar timelocal( 0, 0, 0, $mday_start, ( $mth_start - 1 ),
			$year_start );
		$unix_time_end =
		  scalar timelocal( 0, 0, 0, $mday_end, ( $mth_end - 1 ), $year_end );

		#find the week number of the year
		#  Week_of_Year
		# ($week,$year) = Week_of_Year($year,$month,$day); # RECOMMENDED
		# $week = Week_of_Year($year,$month,$day);         # DANGEROUS
		# my ($week_start, $year_start_no_use)=
		#                    Week_of_Year($year_start,$mth_start,$mday_start);
		#my ($week_end, $year_end_no_use)=
		#   Week_of_Year($year_end,$mth_end,$mday_end);
	}

	# find the directories meet the critiria
	my $elapsed = runtime();

	#find(	\&wanted(	\$TargetSTNID,
	#					\$TargetSTN,
	#					\$unix_time_start,
	#					\$unix_time_end,
	#					\@AllFiles),
	# find( \&wanted, $SearchDir );

	find(
		sub {
			push @AllFiles, $File::Find::name
			  if -f 
				  && m/($TargetSTNID)_.*_($TargetSTN)_\d+\.\d+$/i
				  && !(/RFNORM/i)
				  && !(/gold/i);

			#print "#";
			#$op_textRef->insert( 'end', "#");
			#$op_textRef->see('end');
		},
		$SearchDir
	);

	my $sincethen = runinterval(1);
	print "Finding AllFiles: $sincethen seconds\n";
	if ( @AllFiles < 1 ) {
		logmsg( $op_textRef, "No file meets critiria... " );

		# print "No file meets critiria... \n";
		return 0;
	}

	logmsg($op_textRef, "checking the date in file name");
	
	foreach my $index ( 0 .. $#AllFiles ) {
		my $unix_time_curr;
		my $filename = basename( $AllFiles[$index] );
		print "$AllFiles[$index] \n";
		my ( $year_curr, $mth_curr, $mday_curr ) =
		  $AllFiles[$index] =~ /_(\d\d\d\d)(\d\d)(\d\d)\.\d+/;
		$unix_time_curr =
		  scalar timelocal( 0, 0, 0, $mday_curr, $mth_curr - 1, $year_curr );

		if (   $unix_time_curr < $unix_time_start
			&& $unix_time_curr > $unix_time_end )
		{
			print "not in date range, deleting ...";
			delete $AllFiles[$index];
		}
	}

	my $IsTitleDisplayed = 0;
	my @tmpTitle         = ();
	my @tmpSubTitle      = ();
	my @tmpUnit          = ();
	my @tmpLowLimit      = ();
	my @tmpHiLimit       = ();
	my $bReturn;
	my $headerColumnQty;

	foreach (@AllFiles) {
		my $CurrentFile = $_;
		# logmsg( $op_textRef, "$CurrentFile\n" );

		# Get the records which match the date from the data files.
		$bReturn = GetDataFromCurrentFile(
			\$CurrentFile,  \%AllLinesHoAA,   \%CommonFieldIndexH,
			\@FieldIndex,   \@tmpTitle,       \@tmpSubTitle,
			\@tmpUnit,      \@tmpLowLimit,    \@tmpHiLimit,
			\$TargetATEVer, \$TargetSubTitle, \@TestSearchList,
			$op_textRef
		);

		# Do this when returns arrays are fill up and Result array is empty.
		# Just assign once.
		next unless ( $bReturn == 1 );

		if ( @tmpTitle && !@ResultTitles ) {
			@ResultTitles = @tmpTitle;

			# Get the column qty of header.
			# All qty are same, Title, SubTitle, Unit, etc.
			# They were checked in GetDataFromCurrentFile()
			$headerColumnQty = @ResultTitles;
		}
		if ( @tmpSubTitle && !@ResultSubTitles ) {
			@ResultSubTitles = @tmpSubTitle;
		}
		if ( @tmpUnit && !@ResultUnits ) {
			@ResultUnits = @tmpUnit;
		}
		if ( @tmpLowLimit && !@ResultLowLimit ) {
			@ResultLowLimit = @tmpLowLimit;
		}
		if ( @tmpHiLimit && !@ResultHiLimit ) {
			@ResultHiLimit = @tmpHiLimit;
		}
		
		# if the titles qty from current file is not equal to the  header column
		# qty from 1st matching file, return 0
		if ( $headerColumnQty != @tmpTitle ) {
			logmsg( $op_textRef, "\@ResultTitles:" . @ResultTitles );
			logmsg( $op_textRef, "\@ResultSubTitles:" . @ResultSubTitles );
			logmsg( $op_textRef, "\@ResultUnits:	" . @ResultUnits );
			logmsg( $op_textRef, "\@ResultLowLimit:" . @ResultLowLimit );
			logmsg( $op_textRef, "\@ResultHiLimit:	" . @ResultHiLimit );
			
			logmsg( $op_textRef, "The header column qty is not constant," );
			logmsg( $op_textRef,
				"Assigning test software verion should solve this ..." );
			return 0;
		}
	}

	#my $columnqty = @ResultTitles;

	#print "\@ResultTitles: 	" . @ResultTitles . "\n";
	#logmsg( $op_textRef, "\@ResultTitles:" . @ResultTitles );
	#print "\@ResultSubTitles:" . @ResultSubTitles . "\n";
	#logmsg( $op_textRef, "\@ResultSubTitles:" . @ResultSubTitles );
	#print "\@ResultUnits:	" . @ResultUnits . "\n";
	#logmsg( $op_textRef, "\@ResultUnits:	" . @ResultUnits );
	#print "\@ResultLowLimit:" . @ResultLowLimit . "\n";
	#logmsg( $op_textRef, "\@ResultLowLimit:" . @ResultLowLimit );
	#print "\@ResultHiLimit:	" . @ResultHiLimit . "\n";
	#logmsg( $op_textRef, "\@ResultHiLimit:	" . @ResultHiLimit );

	#if (   @ResultTitles != $columnqty
	#	|| @ResultSubTitles != $columnqty
	#	|| @ResultUnits != $columnqty
	#	|| @ResultLowLimit != $columnqty
	#	|| @ResultHiLimit != $columnqty
	#	|| 0 == $columnqty )
	#{
	#	print "data file format is not consistant \n";
	#	logmsg( $op_textRef, "data file format is not consistant" );
	#	print "normally could be fixed by assinging test code version";
	#	print "before search";
	#	exit;
	#}

	$sincethen = runinterval(1);
	print "Get records from \@AllFiles: $sincethen seconds\n";

	# Find the INDEXes for pre-defined fields
	#FindFixFieldsIndex();

	# Get matching records from the files.
	$elapsed = runtime();

	FindMatchingRecords(
		\@ResultLinesAoA,
		\%AllLinesHoAA,
		\%PassedDUTQty,
		\%CommonFieldIndexH,

		\%FirstHoA,
		\%SecondHoA,
		\%FirstOccuranceH,
		\%SecondOccuranceH,
		\%ECOfAllMatchingRecordsH,

		\$FSNPrefix,
		\$ErrorCode,
		\$TargetATEVer,
		\$TargetBOMRev,
		\$StartDate,
		\$EndDate,
		\$DUTFWRevision,
		\$unix_time_start,
		\$unix_time_end,
		\$KeepDateStampOnly,
		\$CombineErrorCode,

		\@ATERevsUsed,
		\@BOMRevsUsed,
		\@DUTFWRevsUsed,
		\@STNsUsed,
		\@DUTPrefixUsed,
		\@ProdIDUsed,
	);

	$sincethen = runinterval(1);
	print "\nFinding Matching record: $sincethen seconds \n";

	if ( !@ResultLinesAoA )    #$AllLinesHoAA
	{
		print "No records returned";
		return;
	}

	print "\n###############################################################\n";

	# display time for matching records.
	$sincethen = runinterval(1);
	print "Printing matching records: $sincethen seconds \n";

	my @buildsummaryAoA;
	my @ECbreakdownAoA;

	SummaryPrint(
		\@ResultLinesAoA,  \@buildsummaryAoA,  \@ECbreakdownAoA,
		\%FirstOccuranceH, \%SecondOccuranceH, \%PassedDUTQty,
		\@ATERevsUsed,     \@BOMRevsUsed,      \@DUTFWRevsUsed,
		\@STNsUsed,        \@DUTPrefixUsed,    \@ProdIDUsed,
		\%CommonFieldIndexH
	);

	my $ParetoChart_DataFile;   # = $workingdir . '/DataSource/ParetoChart.txt';
	    # print "Pareto data file: $pareto_chart_datafile \n";
	my $ParetoChart_PNGFile;

	#print "======= calculate process capability indices =========== \n";
	my $process_capability_datafile =
	  $workingdir . '/DataSource/ProcessCapability.txt';
	my $process_capability_limitfile =
	  $workingdir . '/DataSource/ProcessCapability-limits.txt';

	my @processcapAoA;

	# Clean the output directory
	if ( 1 == $looptest ) {
		GenErrorCodePareto( \%ECOfAllMatchingRecordsH, \$ParetoChart_DataFile,
			\$ParetoChart_PNGFile, \$No_Of_Top_EC, \$workingdir, \$outputdir );

		if ( 1 == $gen_dis_graph ) {
			GenProcessCapabilityChart(
				$op_textRef,
				\@ResultLinesAoA,

				# \$process_capability_datafile,
				# \$process_capability_limitfile,
				\@processcapAoA,
				\@ResultTitles,
				\@ResultSubTitles,
				\@ResultUnits,
				\@ResultLowLimit,
				\@ResultHiLimit,

				\@TestSearchList,
				\%searchlistH,
				\$outputdir,
				\$workingdir,
				\$decimallength,
				\%CommonFieldIndexH
			);
		}
	}
	else {

		# Generate error code pareto chart
		GenErrorCodePareto( \%FirstOccuranceH, \$ParetoChart_DataFile,
			\$ParetoChart_PNGFile, \$No_Of_Top_EC, \$workingdir, \$outputdir );

		my @array_ref;
		my $elapsed = runtime();
		foreach my $ref ( keys %FirstHoA ) {
			push @array_ref, [ @{ $FirstHoA{$ref} } ];
		}

		if ( $gen_dis_graph  =~ /Yes/i) {

			# Generate Process capability chart for all measurement
			GenProcessCapabilityChart(
				$op_textRef,
				\@array_ref,

				# \$process_capability_datafile,
				# \$process_capability_limitfile,
				\@processcapAoA,
				\@ResultTitles,
				\@ResultSubTitles,
				\@ResultUnits,
				\@ResultLowLimit,
				\@ResultHiLimit,

				\@TestSearchList,
				\%searchlistH,
				\$outputdir,
				\$workingdir,
				\$decimallength,
				\%CommonFieldIndexH
			);
		}

		my $sincethen = runinterval(1);
		print "=== Generating capability chart took: $sincethen seconds\n";
	}

	my @summaryCharts;
	push @summaryCharts, $ParetoChart_PNGFile;
	GenHtmlFile(	\$workingdir,
			        \@buildsummaryAoA, 
			        \@ECbreakdownAoA,
					#\$ParetoChart_PNGFile,
					\@summaryCharts, 
					\@processcapAoA,   
					\$outputfile );

	print
	  "\n###################### End of Scrip ###############################\n";
}

################################################################################
#
################################################################################
sub FindFixFieldsIndex {
	my $FieldIndexRef = shift;
	my $Titles        = shift;
	my $tmpindex      = 0;

	my $tmpTitleName;

 #print "item with index %i in list is 4", firstidx { $_ == 'STN_ID' } @$Titles;
 #$tmpTitleName = 'STN_ID';
 #$$FieldIndexRef{$tmpTitleName } = firstidx { $_ == $tmpTitleName } @$Titles;
 #$tmpTitleName = 'Error Code';
 #$$FieldIndexRef{$tmpTitleName } = firstidx { $_ == $tmpTitleName } @$Titles;

	# '@$Titles-1' must be used, don't use '@$Titles' only.
	# It will cross the border of the array when the $idx is used.
	foreach my $idx ( 0 .. @$Titles - 1 ) {

		#print "The \${\$Titles}[$idx]: ${$Titles}[$idx]\n";
		#if (/^(STN_ID)$/i)     { $$FieldIndexRef{'STN_ID'} 		= $idx; next; }

		if ( ${$Titles}[$idx] =~ /^(STN_ID)$/i ) {

			#if ( $Titles->[$idx] =~ /^(STN_ID)$/i ) {
			#print "the \$idx : $idx \n";
			$$FieldIndexRef{'STN_ID'} = $idx;
			next;
		}

		#if (/^(Error Code)$/i) { $$FieldIndexRef{'Error Code'} 	= $idx; next; }
		if ( ${$Titles}[$idx] =~ /^(Error Code)$/i ) {

			#print "the \$idx : $idx \n";
			$$FieldIndexRef{'Error Code'} = $idx;
			next;
		}

		#if (/^(FSN)$/i)        { $$FieldIndexRef{'FSN'} 		= $idx; next; }
		if ( ${$Titles}[$idx] =~ /^(FSN)$/i ) {

			#print "the \${\$Titles}[$idx]: ${$Titles}[$idx]\n";
			$$FieldIndexRef{'FSN'} = $idx;
			next;
		}

		#if (/^(ATE Vers)$/i)   { $$FieldIndexRef{'ATE Vers'} 	= $idx; next; }
		if ( ${$Titles}[$idx] =~ /^(ATE Vers)$/i ) {
			$$FieldIndexRef{'ATE Vers'} = $idx;
			next;
		}

		#if (/^(Modem Type)$/i) { $$FieldIndexRef{'Modem Type'} 	= $idx; next; }
		if ( ${$Titles}[$idx] =~ /^(Modem Type)$/i ) {
			$$FieldIndexRef{'Modem Type'} = $idx;
			next;
		}

		#if (/^(BOM Rev)$/i)    { $$FieldIndexRef{'BOM Rev'} 	= $idx; next; }
		if ( ${$Titles}[$idx] =~ /^(BOM Rev)$/i ) {
			$$FieldIndexRef{'BOM Rev'} = $idx;
			next;
		}

		#if (/^(FW Rev)$/i)     { $$FieldIndexRef{'FW Rev'} 		= $idx; next; }
		if ( ${$Titles}[$idx] =~ /^(FW Rev)$/i ) {
			$$FieldIndexRef{'FW Rev'} = $idx;
			next;
		}

		#if (/^(Start Time)$/i) { $$FieldIndexRef{'Start Time'}	= $idx; next; }
		if ( ${$Titles}[$idx] =~ /^(Start Time)$/i ) {
			$$FieldIndexRef{'Start Time'} = $idx;
			next;
		}
	}
}
################################################################################
#
################################################################################
sub FindMatchingRecords {
	my $ResultLinesAoARef = shift;
	my $AllLinesHoAARef   = shift;
	my $PassedDUTQtyRef   = shift;
	my $FieldIndexHRef    = shift;

	my $FirstHoARef  = shift;
	my $SecondHoARef = shift;

	my $FirstOccuranceHRef  = shift;
	my $SecondOccuranceHRef = shift;

	my $ECOfAllMatchingRecordsHRef = shift;

	my $FSNPrefixRef        = shift;
	my $ErrorCodeRef        = shift;
	my $TargetATEVerRef     = shift;
	my $TargetBOMRevRef     = shift;
	my $StartDateRef        = shift;
	my $EndDateRef          = shift;
	my $FWVersionRef        = shift;
	my $TimeStartRef        = shift;
	my $TimeEndRef          = shift;
	my $KeepDateStampRef    = shift;
	my $CombineErrorCodeRef = shift;

	my $ATERevsUsedRef   = shift;
	my $BOMRevsUsedRef   = shift;
	my $DUTFWRevsUsedRef = shift;
	my $STNsUsedRef      = shift;

	my $DUTPrefixUsedRef = shift;
	my $ProdIDUsedRef    = shift;

	# The results will be sorted.
  MAINLOOP:

	#foreach my $timekeys ( sort keys %AllLinesHoAA )
	foreach my $timekeys ( sort keys %$AllLinesHoAARef )
	{    # convert FSN to upper case, otherwise it will cause miscount for
		    # Unique FSN. if exist won't ignore the different case?
		    #${ $AllLinesHoAA{$timekeys} }[$FSN_INDEX] =
		${ $$AllLinesHoAARef{$timekeys} }[ $$FieldIndexHRef{'FSN'} ] =
		  uc( ${ $$AllLinesHoAARef{$timekeys} }[ $$FieldIndexHRef{'FSN'} ] );

	 # if the unique records required.
	 # check if the result Array has the current FSN already.
	 # bug fix: the modifier 'i' must be used. otherwise the lower case FSN will
	 # be re-counted.

		# matach the FSN
		next
		  unless (
			${ $$AllLinesHoAARef{$timekeys} }[ $$FieldIndexHRef{'FSN'} ] =~
			/$$FSNPrefixRef/i );

		# Match the Error code
		next
		  unless (
			${ $$AllLinesHoAARef{$timekeys} }[ $$FieldIndexHRef{'Error Code'} ]
			=~ /$$ErrorCodeRef/i );

		# match the model number in records.
		# can't match the model no here. if the DUT was not turned on,
		# there won't be any model no in the record. so all error code occured
		# before power on, will be ignored.

		#next unless (( ${$AllLinesHoAA{$timekeys}}[$MODELNO_INDEX+1] =~
		#        /^$TargetModelInRecord/i) || ($TargetModelInRecord =~ /^$/) );

		# Match the ATE Version
		next
		  unless (
			${ $$AllLinesHoAARef{$timekeys} }[ $$FieldIndexHRef{'ATE Vers'} ] =~
			/^$$TargetATEVerRef$/i );

		# Match the BOM Rev in FSN number. match the one in FSN.
		next
		  unless (
			substr(
				${ $$AllLinesHoAARef{$timekeys} }[ $$FieldIndexHRef{'FSN'} ],
				13, 2 ) =~ /$$TargetBOMRevRef/i
		  );

		# Match FW version
		next
		  unless (
			${ $$AllLinesHoAARef{$timekeys} }[ $$FieldIndexHRef{'FW Rev'} ] =~
			/^$$FWVersionRef$/i );

		# Just search the records in between the date assigned.
		if ( ( $$StartDateRef !~ /^$/ ) && ( $$EndDateRef !~ /^$/ ) ) {
			my ( $mday_curr, $mth_curr, $year_curr ) =
			  ${ $$AllLinesHoAARef{$timekeys} }
			  [ $$FieldIndexHRef{'Start Time'} ] =~ /(\d+)\/(\d+)\/(\d+)/;
			my $unix_time_curr =
			  scalar timelocal( 0, 0, 0, $mday_curr, $mth_curr - 1,
				$year_curr );

			next
			  unless ( $unix_time_curr >= $$TimeStartRef
				&& $unix_time_curr <= $$TimeEndRef );
		}
		############################################################################
		################## End of filtering ########################################
		############################################################################
		# collect all ATE Revisions.
		if (
			not grep(
				/^$$AllLinesHoAARef{$timekeys}[$$FieldIndexHRef{'ATE Vers'}]$/i,
				@$ATERevsUsedRef )
		  )
		{
			push @$ATERevsUsedRef,
			  ${ $$AllLinesHoAARef{$timekeys} }[ $$FieldIndexHRef{'ATE Vers'} ];
		}

   # collect all BOM revisions @BOMRevsUsed and DUT Frepfix,     #@DUTPrefixUsed
   # can't put the substr function in the grep() function. ?
		my $tmpBOM =
		  substr( $$AllLinesHoAARef{$timekeys}[ $$FieldIndexHRef{'FSN'} ],
			13, 2 );
		my $tmpDUTPrefix =
		  substr( $$AllLinesHoAARef{$timekeys}[ $$FieldIndexHRef{'FSN'} ],
			0, 1 );
		my $tmpProdID =
		  substr( $$AllLinesHoAARef{$timekeys}[ $$FieldIndexHRef{'FSN'} ],
			1, 2 );
		if ( not grep( /$tmpBOM/i, @$BOMRevsUsedRef ) ) {
			push @$BOMRevsUsedRef, $tmpBOM;
		}
		if ( not grep( /$tmpDUTPrefix/i, @$DUTPrefixUsedRef ) ) {
			push @$DUTPrefixUsedRef, $tmpDUTPrefix;
		}

		#@ProdIDUsed
		if ( not grep( /$tmpProdID/i, @$ProdIDUsedRef ) ) {
			push @$ProdIDUsedRef, $tmpProdID;
		}

		#@DUTFWRevsUsed
		if (
			not
			grep( /$$AllLinesHoAARef{$timekeys}[$$FieldIndexHRef{'FW Rev'}]/i,
				@$DUTFWRevsUsedRef )
		  )
		{
			push @$DUTFWRevsUsedRef,
			  ${ $$AllLinesHoAARef{$timekeys} }[ $$FieldIndexHRef{'FW Rev'} ];
		}

		#@STNsUsed
		#if (not grep(/${$AllLinesHoAA{$timekeys}}[$STNID_INDEX]/i,@STNsUsed))
		if (
			not
			grep( /$$AllLinesHoAARef{$timekeys}[$$FieldIndexHRef{'STN_ID'}]/i,
				@$STNsUsedRef )
		  )
		{
			push @$STNsUsedRef,
			  ${ $$AllLinesHoAARef{$timekeys} }[ $$FieldIndexHRef{'STN_ID'} ];
		}

		# remove the time string from the 'Start_Time' field.
		if ( 0 == $$KeepDateStampRef ) {
			( ${ $$AllLinesHoAARef{$timekeys} }
				  [ $$FieldIndexHRef{'Start Time'} ] ) =
			  ${ $$AllLinesHoAARef{$timekeys} }
			  [ $$FieldIndexHRef{'Start Time'} ] =~ /(\d\d\/\d\d\/\d\d\d\d)/;
		}

		# Choose the right ErrorCode.
		my $ErrorCodeKey;
		my $ErrorCodeBand;
		if ( 1 == $$CombineErrorCodeRef ) {
			( $ErrorCodeKey, $ErrorCodeBand ) =
			  ${ $$AllLinesHoAARef{$timekeys} }
			  [ $$FieldIndexHRef{'Error Code'} ] =~ /(\w+)-?(\w*)/;

			# get rid of the band infor from error code.
			${ $$AllLinesHoAARef{$timekeys} }[ $$FieldIndexHRef{'Error Code'} ]
			  = $ErrorCodeKey;
		}
		else {
			$ErrorCodeKey =
			  ${ $$AllLinesHoAARef{$timekeys} }
			  [ $$FieldIndexHRef{'Error Code'} ];
		}

		# substitute SKIP with * for Minitab data analysis.
		foreach ( @{ $$AllLinesHoAARef{$timekeys} } ) { s/(SKIP)/*/g }

		# Push whole record matching critiria into   @ResultLinesAoA
		# include date, FSN, BOM, FW Rev, ErrorCode, Model No, ATE Ver.
		push @$ResultLinesAoARef, [ @{ $$AllLinesHoAARef{$timekeys} } ];

		# Count the Passed DUTs
		# %AllLinesHoAA contains all record between $StartDate and $EndDate

		#if ( $ResultLinesAoA[-1][$ERRORCODE_INDEX] =~ /(OK)/i ) {
		if ( $$ResultLinesAoARef[-1][ $$FieldIndexHRef{'Error Code'} ] =~
			/(OK)/i )
		{

			#$PassedDUTQty{ $ResultLinesAoA[-1][$FSN_INDEX] } =
			#$PassedDUTQty{ ${@$ResultLinesAoARef}[-1][$FSN_INDEX] } =
			$$PassedDUTQtyRef{ $$ResultLinesAoARef[-1]
				  [ $$FieldIndexHRef{'FSN'} ] } =

			  #$ResultLinesAoA[-1][$ERRORCODE_INDEX];
			  $$ResultLinesAoARef[-1][ $$FieldIndexHRef{'Error Code'} ];
		}

		# get error code qty for all matching records.
		#$ECOfAllMatchingRecordsH{$ErrorCodeKey} += 1;
		$$ECOfAllMatchingRecordsHRef{$ErrorCodeKey} += 1;

		# Find the 1st and 2nd occurance in  %AllLinesHoAA.
		#if ( exists $FirstHoA{ $ResultLinesAoA[-1][$FSN_INDEX] } ) {
		if (
			exists
			$$FirstHoARef{ $$ResultLinesAoARef[-1][ $$FieldIndexHRef{'FSN'} ] }
		  )
		{

			# if DUT passed 1st time, the $SecondHoA won't count that.
			if (
				not exists $$SecondHoARef{ $$ResultLinesAoARef[-1]
					  [ $$FieldIndexHRef{'FSN'} ] } )
			{

				#if ( $FirstHoA{ $ResultLinesAoA[-1][$FSN_INDEX] }
				if (
					$$FirstHoARef{ $$ResultLinesAoARef[-1]
						  [ $$FieldIndexHRef{'FSN'} ] }
					[ $$FieldIndexHRef{'Error Code'} ] !~ /^OK$/ )
				{

		  #push @{ $SecondHoA{ $ResultLinesAoA[-1][$$FieldIndexHRef{'FSN'}] } },
					push @{
						$$SecondHoARef{
							$$ResultLinesAoARef[-1][ $$FieldIndexHRef{'FSN'} ]
						  }
					  },

					  #@{ $ResultLinesAoA[-1] };
					  @{ $$ResultLinesAoARef[-1] };

					#$SecondOccuranceH{$ErrorCodeKey} += 1;
					$$SecondOccuranceHRef{$ErrorCodeKey} += 1;
				}
			}
		}
		else {

  # push @{$FirstHoA{ $ResultLinesAoA[-1][$FSN_INDEX]}}, @{$ResultLinesAoA[-1]};
			push @{ $$FirstHoARef{ $$ResultLinesAoARef[-1]
					  [ $$FieldIndexHRef{'FSN'} ] } },
			  @{ $$ResultLinesAoARef[-1] };

			#$FirstOccuranceH{$ErrorCodeKey} += 1;
			$$FirstOccuranceHRef{$ErrorCodeKey} += 1;
		}
	}
}

################################################################################
#
################################################################################
#sub wanted {
#	my $StnID     = shift;
#	my $StnType   = shift;
#	my $TimeStart = shift;
#	my $TimeEnd   = shift;
#	my $Files     = shift;
#
#	my $rindextmp;
#	my $unix_time_curr;
#	my $stationID   = "BFT";
#	my $stationType = ".*";
#
#	# Find the wanted file names, make sure isnot case sensitivity  $TargetDate
#	if (
#		$File::Find::name =~ m/($stationID)_.*_($stationType)_\d+\.\d+$/i
#
#		# Match the STN Type and staiton ID.
#		&& !(/RFNORM/i)    # doesn't match NORM files
#		&& !(/gold/i)
#	  )                    # Doesn't match the gold
#	{
#		print "searching('\$File::Find::name') ...", $File::Find::name, "\n";
#
#		#print "searching('\$_') ...",$_, "\n";
#		my ( $year_curr, $mth_curr, $mday_curr ) =
#		  $_ =~ /_(\d\d\d\d)(\d\d)(\d\d)\.\d+/;
#		$unix_time_curr =
#		  scalar timelocal( 0, 0, 0, $mday_curr, $mth_curr - 1, $year_curr );
#
#	  #print "day, month, year=",$mday_curr,'-',$mth_curr-1,'-',$year_curr,"\n";
#		if ( $unix_time_curr >= $$TimeStart && $unix_time_curr <= $$TimeEnd ) {
#
#			#push( @AllFiles, $File::Find::name );
#			push( @$Files, $File::Find::name );
#		}
#	}
#}
################################################################################
#
################################################################################
sub GetDataFromCurrentFile {
	my $FileRef             = shift;
	my $AllLinesHoAARef     = shift;
	my $CommonFieldIndexRef = shift;

	my $FieldIndexRef      = shift;
	my $ResultTitlesRef    = shift;
	my $ResultSubTitlesRef = shift;
	my $ResultUnitsRef     = shift;
	my $ResultLowLimitRef  = shift;
	my $ResultHiLimitRef   = shift;
	my $ATEVersionRef      = shift;
	my $TargetSubTitleRef  = shift;
	my $TitleSearchListRef = shift;
	my $op_textRef         = shift;

	my @Titles;
	my @SubTitles;
	my @Units;
	my @LowLimit;
	my @HiLimit;
	my @CurrentLineTmp;

	my $StationID = '';    # Hold the station ID from the data file.

	#print "\$FileRef			= $$FileRef	\n";
	#print "\$AllLinesHoAARef	= %$AllLinesHoAARef	\n";
	#print "\$FieldIndexRef		= @$FieldIndexRef	\n";
	#print "\$ResultTitlesRef	= @$ResultTitlesRef	\n";
	#print "\$ResultSubTitlesRef	= @$ResultSubTitlesRef	\n";
	#print "\$ResultUnitsRef		= @$ResultUnitsRef		\n";
	#print "\$ResultLowLimitRef	= @$ResultLowLimitRef	\n";
	#print "\$ResultHiLimitRef	= @$ResultHiLimitRef	\n";
	#print "\$ATEVersionRef		= $$ATEVersionRef		\n";
	#print "\$TargetSubTitleRef	= $$TargetSubTitleRef	\n";
	#print "$TitleSearchListRef	= @$TitleSearchListRef	\n";

	# Open file handler.
	open( CURRENTFILE, $$FileRef ) or die "can't fopen input file";

	$$FileRef =~ s/\//\\/g;
	print("Processing: $$FileRef \n");

	my @Array_currentFile = <CURRENTFILE>;
	close(CURRENTFILE);

	my $linecounter = 0;
	foreach (@Array_currentFile) {
		$linecounter++;
		my $ateversion;
		chomp;    # remove the return?
		          # if line starts with 'Key'
		next if (/^(Key)/);

		if (/^(ATE)/) {
			($ateversion) = $_ =~ /Version(.+)$/i;
			if ( $ateversion !~ /$$ATEVersionRef/ ) {
				logmsg($op_textRef, "ATE version doesn't match, next file...");
				return 0;    # return from current file.
			}
			next;
		}

		# Get the station ID
		if (/^(Test Station)/) {
			( my $tmp, $StationID ) = split(/:/);

			$StationID =~ s/^\s+|\s+$//g;
			next;
		}

		if ( 4 == $linecounter && /^(FSN)/ && !@Titles ) {
			@Titles = split(/,/);
			foreach (@Titles) { s/^\s+|\s+$//g; }

			next;
		}

		if ( 5 == $linecounter && /^,+/ && !@SubTitles ) {
			@SubTitles = split(/,/);
			foreach (@SubTitles) { s/^\s+|\s+$//g; }
			next;
		}

		if ( 6 == $linecounter && /^,+/ && !@Units ) {
			@Units = split(/,/);
			foreach (@Units) { s/^\s+|\s+$//g; }
			next;
		}

		if ( 7 == $linecounter && /^,+/ && !@LowLimit ) {
			@LowLimit = split(/,/);
			foreach (@LowLimit) { s/^\s+|\s+$//g; }
			next;
		}

		if ( 8 == $linecounter && /^,+/ && !@HiLimit ) {
			@HiLimit = split(/,/);
			foreach (@HiLimit) { s/^\s+|\s+$//g; }
			next;
		}

		# This will skip some wrong foramt data files without Station ID.
		if ( $StationID =~ /^$/i ) {
			print "\nSkip file: $$FileRef\n";

			#last;    # parse next file.
			return 0;    # parse next file.
		}

		# Check only once after all headers are extracted
		if ( 9 == $linecounter ) {

			@$FieldIndexRef      = ();
			@$ResultTitlesRef    = ();
			@$ResultSubTitlesRef = ();
			@$ResultUnitsRef     = ();
			@$ResultLowLimitRef  = ();
			@$ResultHiLimitRef   = ();

			push @$ResultTitlesRef,    "STN_ID";
			push @$ResultTitlesRef,    @Titles;
			push @$ResultSubTitlesRef, "";
			push @$ResultSubTitlesRef, @SubTitles;
			push @$ResultUnitsRef,     "";
			push @$ResultUnitsRef,     @Units;
			push @$ResultLowLimitRef,  "";
			push @$ResultLowLimitRef,  @LowLimit;
			push @$ResultHiLimitRef,   "";
			push @$ResultHiLimitRef,   @HiLimit;

			FindFixFieldsIndex( $CommonFieldIndexRef, $ResultTitlesRef );
			GetFieldIndex(
				$FieldIndexRef,      $ResultTitlesRef, $ResultSubTitlesRef,
				$TitleSearchListRef, $TargetSubTitleRef
			);

			logmsg( $op_textRef, "\@ResultTitlesRef Qty:" . @$ResultTitlesRef );
			logmsg( $op_textRef,
				"\@ResultSubTitlesRef Qty:" . @$ResultSubTitlesRef );
			logmsg( $op_textRef, "\@ResultUnitsRef Qty:" . @$ResultUnitsRef );
			logmsg( $op_textRef,
				"\@ResultLowLimitRef Qty:" . @$ResultLowLimitRef );
			logmsg( $op_textRef,
				"\@ResultHiLimitRef Qty:" . @$ResultHiLimitRef );

			# Step 1 of 2:  Check if title fields are equal, if not, return 0
			if (   ( @$ResultTitlesRef != @$ResultSubTitlesRef )
				&& ( @$ResultTitlesRef != @$ResultUnitsRef )
				&& ( @$ResultTitlesRef != @$ResultLowLimitRef )
				&& ( @$ResultTitlesRef != @$ResultHiLimitRef ) )
			{
				$#Titles    = -1;
				$#SubTitles = -1;
				$#Units     = -1;
				$#LowLimit  = -1;
				$#HiLimit   = -1;
				logmsg( $op_textRef,
					"Title filed qty are not equal, continue with next file" );
				return 0;
			}
		}
		my $TitleColumnQty = @$ResultTitlesRef;

		# After getting all title, subtitle, unit, low/hi limit lines,
		# Start processing the records.
		# Match any string start with any letter plus a 10 digits number
		next unless (m/^\w{3}\d{8}/i);

		# Split the current line. it includes all fields.
		@CurrentLineTmp = split(/,/);

		# Insert $StationID as first element in the array
		splice( @CurrentLineTmp, 0, 0, $StationID );

		# Step 2 of 2: Check if record column equals title filed column.
		#if ( 9 == $linecounter
		#	&& $CurrentLineTmp[ $$CommonFieldIndexRef{'Error Code'} ] !~ /OK/ )
		if ( $TitleColumnQty != @CurrentLineTmp ) {
			logmsg( $op_textRef,
"Title column qty and record column qty are not equal, continue with next file"
			);

			# Continue with next line
			next;
		}

		#######################################################
		# Make the output unique
		#######################################################
		my ( $mday, $mth, $year ) =
		  $CurrentLineTmp[ $$CommonFieldIndexRef{'Start Time'} ] =~
		  /(\d+)\/(\d+)\/(\d+)/;
		my ( $hour, $min, $sec ) =
		  $CurrentLineTmp[ $$CommonFieldIndexRef{'Start Time'} ] =~
		  /(\d+):(\d+):(\d+)/;
		my $unix_time =
		  scalar timelocal( $sec, $min, $hour, $mday, ( $mth - 1 ), $year );

		# check the time stamp to see if same record exists already.
		while ( @$AllLinesHoAARef{$unix_time} )
		{    # if record exists, then check if existed record has same FSN
			    # if exist record has same FSN, it's just duplicated record.
			    # continue with next line from current file.
			 # if($AllLinesHoAA{$unix_time}[1] =~ /$TmpLine[1]/i) {goto NextLine; }
			 #if ( $AllLinesHoAA{$unix_time}[1] =~ /$CurrentLineTmp[0]/i ) {
			if ( $$AllLinesHoAARef{$unix_time}[1] =~ /$CurrentLineTmp[0]/i ) {
				goto NextLine;
			}

			# if current FSN is different from exist FSN in %AllLineHoAA,
			# the $unix_time will be increased.
			$unix_time += 1;
		}
		push @{ $$AllLinesHoAARef{$unix_time} }, @CurrentLineTmp;
	  NextLine:
	}

	return 1;
}

################################################################################
# Get the field index want to be displayed
################################################################################
sub GetFieldIndex {
	my $FieldIndexRef      = shift;
	my $Titles             = shift;
	my $SubTitles          = shift;
	my $TitleSearchListRef = shift;
	my $TargetSubTitleRef  = shift;

	my $i = 0;

	#foreach (@TestSearchList) {
	foreach (@$TitleSearchListRef) {
		my $FieldIndextmp = 0;

		#foreach (@ResultTitles) {
		foreach (@$Titles) {

			#if (/^$TestSearchList[$i]$/i) {
			if (/^${$TitleSearchListRef}[$i]$/i) {

			   #if ( ( $ResultSubTitles[$FieldIndextmp] =~ m/$TargetSubTitle/i )
				if (
					(
						${$SubTitles}[$FieldIndextmp] =~ m/$$TargetSubTitleRef/i
					)

					#|| ( $ResultSubTitles[$FieldIndextmp] eq '' ) )
					|| ( ${$SubTitles}[$FieldIndextmp] eq '' )
				  )

				  #|| ( $TargetSubTitle eq '' ) )
				{    # if the Title match the item from the  @TestSearchList
					    #push( @FieldIndex, $FieldIndextmp );
					push( @{$FieldIndexRef}, $FieldIndextmp );
				}
			}
			$FieldIndextmp++;
		}
		$i++;
	}
}

################################################################################
# Get the field index want to be displayed
################################################################################
sub SummaryPrint {
	my $ResultLinesAoA_Ref = shift;
	my $buildsummaryAoA    = shift;
	my $ECbreakdownAoA     = shift;

	my $FirstOccuranceH_Ref  = shift;
	my $SecondOccuranceH_Ref = shift;

	my $PassedDUTQtyRef = shift;

	my $ATERevsUsedRef   = shift;
	my $BOMRevsUsedRef   = shift;
	my $DUTFWRevsUsedRef = shift;
	my $STNsUsedRef      = shift;
	my $DUTPrefixUsedRef = shift;
	my $ProdIDUsedRef    = shift;
	my $FieldIndexHRef   = shift;

	my $DataTime             = scalar( localtime() );
	my $No_Of_Unique_FSN_1st = 0;
	my $No_Of_Unique_FSN_2nd = 0;

	#foreach my $keys ( keys %FirstOccuranceH ) {
	foreach my $keys ( keys %{$FirstOccuranceH_Ref} ) {

		#$No_Of_Unique_FSN_1st += $FirstOccuranceH{$keys};
		$No_Of_Unique_FSN_1st += $$FirstOccuranceH_Ref{$keys};
	}

	#foreach my $keys ( keys %SecondOccuranceH ) {
	foreach my $keys ( keys %{$SecondOccuranceH_Ref} ) {

		#$No_Of_Unique_FSN_2nd += $SecondOccuranceH{$keys};
		$No_Of_Unique_FSN_2nd += $$SecondOccuranceH_Ref{$keys};
	}

	# return if ( ($No_Of_Unique_FSN_1st == 0)||($No_Of_Unique_FSN_2nd==0));
	return if ( ( $No_Of_Unique_FSN_1st == 0 ) );

# initalize there is not DUT passed.
# if (not exists $FirstOccuranceH{'OK'}) {$FirstOccuranceH{'OK'}=0; }
# return unless ( defined($No_Of_Unique_FSN_1st) && ($No_Of_Unique_FSN_1st>1) );

	print "\n\n###################### Sumamry #########################\n";
	printf "%12s,%12s,%12s,%12s,%12s, \n",
	  "ErrorCode", "1st_Test", "1st_Test%", "2nd_Test",
	  "2nd_Test_%";
	foreach my $keys (

		#sort { $FirstOccuranceH{$b} <=> $FirstOccuranceH{$a} }
		sort { ${$FirstOccuranceH_Ref}{$b} <=> ${$FirstOccuranceH_Ref}{$a} }

		#keys %FirstOccuranceH
		keys %{$FirstOccuranceH_Ref}
	  )
	{

		#if ( not exists $SecondOccuranceH{$keys} ) {
		if ( not exists ${$SecondOccuranceH_Ref}{$keys} ) {

			#$SecondOccuranceH{$keys} = 0;
			${$SecondOccuranceH_Ref}{$keys} = 0;
		}
		printf "%12s,", $keys;

		#printf "%11d,", $FirstOccuranceH{$keys};
		printf "%11d,", ${$FirstOccuranceH_Ref}{$keys};
		printf "%10.2f%%,",

		  #$FirstOccuranceH{$keys} * 100 / $No_Of_Unique_FSN_1st;
		  ${$FirstOccuranceH_Ref}{$keys} * 100 / $No_Of_Unique_FSN_1st;

		# printf "%12d,", $SecondOccuranceH{$keys};
		printf "%12d,", ${$SecondOccuranceH_Ref}{$keys};

 # printf "%10.2f%%,\n", $SecondOccuranceH{$keys} * 100 / $No_Of_Unique_FSN_1st;
		printf "%10.2f%%,\n",
		  ${$SecondOccuranceH_Ref}{$keys} * 100 / $No_Of_Unique_FSN_1st;

		push @$ECbreakdownAoA, [
			$keys,

			#$FirstOccuranceH{$keys},
			${$FirstOccuranceH_Ref}{$keys},
			(
				int(
					${$FirstOccuranceH_Ref}{$keys} * 100 * 100 /
					  $No_Of_Unique_FSN_1st
				  ) / 100
			  )
			  . "%",

			#$SecondOccuranceH{$keys},
			${$SecondOccuranceH_Ref}{$keys},

	#(int($SecondOccuranceH{$keys} * 100 * 100 / $No_Of_Unique_FSN_1st)/100)."%"
			(
				int(
					${$SecondOccuranceH_Ref}{$keys} * 100 * 100 /
					  $No_Of_Unique_FSN_1st
				  ) / 100
			  )
			  . "%"
		];

	}
	printf "%12s, %10d, %10.2f%% \n ", 'Total',
	  $No_Of_Unique_FSN_1st,
	  $No_Of_Unique_FSN_1st * 100 / $No_Of_Unique_FSN_1st;

	push @$ECbreakdownAoA,
	  [
		'Total', $No_Of_Unique_FSN_1st,
		$No_Of_Unique_FSN_1st * 100 / $No_Of_Unique_FSN_1st
	  ];

# return if ( ($No_Of_Unique_FSN_2nd == 0));
#	if ($No_Of_Unique_FSN_2nd != 0)
#	{
#		print "\n******** 2nd Occurance *************** \n";
#		printf  "%12s,%12s,%12s, \n",
#		"ErrorCode","2nd_Test", "2nd_Test_%"; # "NTF%_In_Total_FPY";
#		foreach my $keys (sort { $SecondOccuranceH{$b} <=> $SecondOccuranceH{$a} }
#		                                                        keys %SecondOccuranceH)
#		{
#			next if ($SecondOccuranceH{$keys} == 0);
#			printf "%12s,", $keys;
#			printf "%11d,", $SecondOccuranceH{$keys};
#			printf "%10.2f%%, \n", $SecondOccuranceH{$keys}*100/$No_Of_Unique_FSN_2nd;
#		}
#		printf "%12s, %10d, %10.2f%% \n ", 'Total',
#		$No_Of_Unique_FSN_2nd,$No_Of_Unique_FSN_2nd*100/$No_Of_Unique_FSN_2nd ;
#	}

	my $TimeStampFirst;
	my $TimeStampLast;

	# Find out the 1st and last time stamp from records.
	#$TimeStampFirst = $ResultLinesAoA[0][$DATE_INDEX];
	$TimeStampFirst =
	  ${$ResultLinesAoA_Ref}[0][ $$FieldIndexHRef{'Start Time'} ];

	#$TimeStampLast  = $ResultLinesAoA[$#ResultLinesAoA][$DATE_INDEX];
	$TimeStampLast =
	  ${$ResultLinesAoA_Ref}[ $#{$ResultLinesAoA_Ref} ]
	  [ $$FieldIndexHRef{'Start Time'} ];

	# Find out how many 'pass' records.
	my $TotalQtyPassed;

	#for my $key ( keys %PassedDUTQty ) {
	for my $key ( keys %{$PassedDUTQtyRef} ) {

		#if ( $PassedDUTQty{$key} =~ /(OK)/i ) { $TotalQtyPassed++ }
		if ( ${$PassedDUTQtyRef}{$key} =~ /(OK)/i ) { $TotalQtyPassed++ }
	}

	# initialize the $TotalQtyPassed if it's not defined.
	if ( not defined $TotalQtyPassed ) { $TotalQtyPassed = 0; }

	# print the summary
	my $SpaceQty = "%22s";
	print
	  "\n#################################################################\n";
	printf "$SpaceQty, %i \n", 'Qty Tested', $No_Of_Unique_FSN_1st;
	push @$buildsummaryAoA, [ 'Qty Tested', $No_Of_Unique_FSN_1st ];

	#my $tmp = (int($FirstOccuranceH{'OK'}*100/$No_Of_Unique_FSN_1st)*100)/100;
	my $tmp = (
		int( ${$FirstOccuranceH_Ref}{'OK'} * 100 / $No_Of_Unique_FSN_1st ) *
		  100 ) / 100;

	printf "$SpaceQty, %-3.2f%%  \n", 'First_Pass_Yield', $tmp;
	push @$buildsummaryAoA, [ 'First_Pass_Yield', $tmp . '%' ];

#$tmp = ( int(($FirstOccuranceH{'OK'}+$SecondOccuranceH{"OK"})*100/$No_Of_Unique_FSN_1st)*100)/100;
	$tmp = (
		int(
			( ${$FirstOccuranceH_Ref}{'OK'} + ${$SecondOccuranceH_Ref}{"OK"} ) *
			  100 /
			  $No_Of_Unique_FSN_1st
		  ) * 100
	) / 100;

	printf "$SpaceQty, %-3.2f%%  \n", 'Yield_1st+2nd', $tmp;
	push @$buildsummaryAoA, [ 'Yield_1st+2nd', $tmp . '%' ];

	$tmp = ( int( $TotalQtyPassed * 100 / $No_Of_Unique_FSN_1st ) * 100 ) / 100;
	printf "$SpaceQty, %-3.2f%%  \n", 'Overall Yield', $tmp;
	push @$buildsummaryAoA, [ 'Overall Yield', $tmp . '%' ];

	printf "$SpaceQty, %s \n", 'StartDate', $TimeStampFirst;
	push @$buildsummaryAoA, [ 'StartDate', $TimeStampFirst ];

	printf "$SpaceQty, %s \n", 'EndDate', $TimeStampLast;
	push @$buildsummaryAoA, [ 'EndDate', $TimeStampLast ];

	printf "$SpaceQty,\n", 'TestCode Rev';

	#foreach (@ATERevsUsed) { print "\t\t\t,$_ \n"; }
	foreach (@$ATERevsUsedRef) { print "\t\t\t,$_ \n"; }

	#push @$buildsummaryAoA, [ 'TestCode Rev', \@ATERevsUsed ];
	push @$buildsummaryAoA, [ 'TestCode Rev', $ATERevsUsedRef ];

	printf "$SpaceQty,\n", 'BOM Revs';
	foreach (@$BOMRevsUsedRef) { print "\t\t\t,$_ \n"; }
	push @$buildsummaryAoA, [ 'BOM Revs', $BOMRevsUsedRef ];

	printf "$SpaceQty,\n", 'DUT FW Revs';
	foreach (@$DUTFWRevsUsedRef) { print "\t\t\t,$_ \n"; }
	push @$buildsummaryAoA, [ 'DUT FW Revs', $DUTFWRevsUsedRef ];

	printf "$SpaceQty,\n", 'STN Used';
	foreach (@$STNsUsedRef) { print "\t\t\t,$_ \n"; }
	push @$buildsummaryAoA, [ 'STN Used', $STNsUsedRef ];

	printf "$SpaceQty,\n", 'Available DUT Prefix';
	foreach (@$DUTPrefixUsedRef) { print "\t\t\t,$_ \n"; }
	push @$buildsummaryAoA, [ 'Available DUT Prefix', $DUTPrefixUsedRef ];

	printf "$SpaceQty,\n", 'Available Prod ID';
	foreach (@$ProdIDUsedRef) { print "\t\t\t,$_ \n"; }

	push @$buildsummaryAoA, [ 'Available Prod ID', $ProdIDUsedRef ];

	printf "$SpaceQty,\t%s \n\n", 'Date&Time', $DataTime;

	#push @{$summaryHoAref->{'Date&Time'}}, $DataTime;
	push @$buildsummaryAoA, [ 'Date&Time', $DataTime ];

}

################################################################################
# Get the field index want to be displayed
################################################################################
sub GenErrorCodePareto {
	my $hashref         = shift;
	my $DataFileRef     = shift;
	my $PNGFilePathRef  = shift;
	my $No_Of_Top_ECRef = shift;
	my $WorkingDirRef   = shift;
	my $OutPutDirRef    = shift;

	my $data;
	my $labels;
	my @Label_array_of_FirstOccuranceH;
	my @Data_array_of_FirstOccuranceH;

	chdir($OutPutDirRef);
	my $DatasourceDir = $$OutPutDirRef . "/images";
	unless ( -d "$DatasourceDir" ) { mkpath("$DatasourceDir"); }

	$$DataFileRef    = "./images" . "/ParetoChart.txt";
	$$PNGFilePathRef = "./images/paretochartplot.png";
	if ( keys(%$hashref) <= 1 ) {
		print "too less data generate pareto chart \n";
		for my $key ( keys %$hashref ) {
			print "$key \t $hashref->{$key} \n";
		}
		return;
	}

	my $m = 0;
	my $tmpFileName = File::Spec->catfile( $$OutPutDirRef, $$DataFileRef );

	sysopen( FH, $tmpFileName, O_WRONLY | O_TRUNC | O_CREAT ) || die $!;
	print FH "ErrorCode \t Frequency \n";

#LINE: foreach my $keys ( sort { $hashref->{$b} <=> $hashref->{$a} } keys %$hashref )
	foreach
	  my $keys ( sort { $hashref->{$b} <=> $hashref->{$a} } keys %$hashref )
	{
		next if ( $keys =~ /^OK$/ );
		last if ( $m == $$No_Of_Top_ECRef );
		print FH "$keys \t $hashref->{$keys} \n";
		$m += 1;
	}
	close(FH);

   # my $tmpfilename = './DynamicWebPage/images/paretochartplot.png';
   # my $tmpfilename = $$OutPutDirRef . '/images/paretochartplot.png';
   # fcopy( $$WorkingDirRef . "/DynamicWebPage/ParetoChart.png", $tmpfilename );

	my $ret = GenRChart_Pareto( $OutPutDirRef, $DataFileRef, $PNGFilePathRef );

	if   ( $ret == 1 ) { print "pareto chart was created. \n"; }
	else               { print "failed to create pareto chart"; }

	#$$paretochartpathRef = $$OutPutDirRef.'/images/paretochartplot.png';
	# $$paretochartpathRef = './images/paretochartplot.png';
}

####################################################
#
#	Input: 	$DataSetRef
#				- It's a Array of Array. Each elements include the following array.
#				  [0] Column ID, [1] Limit file path [2] Data source file path
####################################################
sub GenRChart_Pareto {
	my $OutPutDirRef   = shift;
	my $DataFileRef    = shift;
	my $PNGFilePathRef = shift;

	my $ReturnFromR;
	my $RCmd;

	chdir("$$OutPutDirRef");
	my $dir = getcwd;
	print "current work directory= $dir, \n\n";

	my $R = Statistics::R->new();
	$R->startR;

	# where to save the PNG file.

	$RCmd = GenRScript_Pareto(

		#$CharttypeRef, 	# Chart type. Pareto Chart, Process Capability chart.
		$OutPutDirRef,     # Current working directory.
		$DataFileRef,      # Data file path.
		$PNGFilePathRef    # PNG file path
	);

	# print "Sending R Script: \n $RCmd \n";
	$R->send(qq($RCmd));

	# my $errR = $R->error;
	# print "Error from R: $errR \n";

	# $R->clean_up();	# Bug? the perl will hangs.

	$ReturnFromR = $R->read;
	print "R Script returns: $ReturnFromR \n";
	$R->stopR();
}

##################################################
#
####################################################
sub GenRScript_Pareto {

	my $OutPutDirRef    = shift;
	my $DataFilePathRef = shift;
	my $PNGFilePathRef  = shift;

	my $RCmd;
	$RCmd = "	# Clean working environment. get rid of leftover
					# rm(list=ls())
					library(qcc)
					graphics.off()
				   	setwd(\"$$OutPutDirRef\")
					png(filename=\"$$PNGFilePathRef\",width = 450, height = 300)
					datasource<-read.table(\"$$DataFilePathRef\",header=TRUE)
					attach(datasource)
					defect<-Frequency
					names(defect)<-ErrorCode
					pareto.chart(defect,col=rainbow(length(defect)))
					dev.off()
					# Clean objects. must be cleaned otherwise the subsquent script won't work properly.
					# it only generate first chart, rest of them are empty image.
					rm(qcc)
					rm(datasource)
					rm(defect)					
				";
	return $RCmd;

}
####################################################
#
#	Input: 	$DataSetRef
#				- It's a Array of Array. Each elements include the following array.
#				  [0] Column ID, [1] Limit file path [2] Data source file path
####################################################
sub GenRChart_ProcessCapability {
	my $op_textRef   = shift;
	my $OutPutDirRef = shift;
	my $CharttypeRef = shift;
	my $DataSetRef   = shift;

	my $ReturnFromR;
	my $RCmd;

	chdir("$$OutPutDirRef");
	my $dir = getcwd;
	print "current work directory= $dir, \n\n";

	my $R = Statistics::R->new();
	$R->startR;
	print "Chart type: $$CharttypeRef \n";

	for my $aref (@$DataSetRef) {

	 # print "Processing: \n Limit file: $aref->[1] \n Data file: $aref->[2]\n";
		logmsg( $op_textRef,
			"Processing: \n Limit file: $aref->[1] \n Data file: $aref->[2]" );
		$RCmd = GenRScript_ProcCapability(
			$CharttypeRef, # Chart type. Pareto Chart, Process Capability chart.
			$OutPutDirRef, # Current working directory.
			\$aref->[1],   # Limit file path
			\$aref->[2],   # Data file path.
			\$aref->[3]    # PNG file path
		);
		$R->send(qq($RCmd));
	}

	$R->stopR();
}

##################################################
#
####################################################
sub GenRScript_ProcCapability {
	my $CharttypeRef = shift;
	my $OutPutDirRef = shift;

	my $LimitFilePathRef = shift;
	my $DataFilePathRef  = shift;
	my $PNGFilePathRef   = shift;

	my $RCmd;

	$RCmd = "	# Clean working environment. get rid of leftover
					# rm(list=ls())
					library(qcc)
					graphics.off()
				   	#setwd(\"$$OutPutDirRef\")
					png(filename=\"$$PNGFilePathRef\",width = 450, height = 300)
					#png(filename=\"./DynamicWebPage/ProcessCapabilityChart.png\",width = 450, height = 300)
					limits<-read.table(\"$$LimitFilePathRef\", header=TRUE,na.strings=\"NA\")
					attach(limits)
					if (is.na(t)) t=NULL
					datasource<-read.table(\"$$DataFilePathRef\", header=TRUE, na.strings=\"NA\");
					attach(datasource)	
					ChartTitle<-names(datasource[1])
					obj<-qcc( datasource,type=\"xbar.one\",target=t,data.name=names(datasource[1]))
					qcc.options(font.stats = 1, cex.stats = 0.8, cex=0.8, bg.margin=\"lightblue\",bg.figure=\"lightyellow\")
					process.capability(obj,spec.limits=c(lsl,usl))
					dev.off()
					# Clean objects. must be cleaned otherwise the subsquent script won't work properly.
					# it only generate first chart, rest of them are empty image.
					rm(limits)
					rm(datasource)
					rm(ChartTitle)
					rm(obj)
					rm(t)
					rm(qcc)
				 ";
	return $RCmd;
}

####################################################
#
#	Input: 	$DataSetRef
#				- It's a Array of Array. Each elements include the following array.
#				  [0] Column ID, [1] Limit file path [2] Data source file path
####################################################
sub GenRChart_Boxplot {
	my $OutPutDirRef = shift;
	my $DataFileRef  = shift;
	my $PNGFileRef   = shift;

	my $ReturnFromR;
	my $RCmd;

	chdir("$$OutPutDirRef");
	my $dir = getcwd;
	print "current work directory= $dir, \n\n";

	my $R = Statistics::R->new();
	$R->startR;

	# print "Chart type: $$CharttypeRef \n";

	$RCmd = GenRScript_BoxPlot(
		$OutPutDirRef,    # Current working directory.
		$DataFileRef,     # Data file path.
		$PNGFileRef       # PNG file path
	);

	# print "Sending R Script: \n $RCmd \n";
	$R->send(qq($RCmd));

	# my $errR = $R->error;
	# print "Error from R: $errR \n";

	# $R->clean_up();	# Bug? the perl will hangs.
	$ReturnFromR = $R->read;
	print "R Script returns: $ReturnFromR \n";
	$R->stopR();
}

##################################################
#
####################################################
sub GenRScript_BoxPlot {
	my $OutPutDirRef    = shift;
	my $DataFilePathRef = shift;
	my $PNGFilePathRef  = shift;

	my $RCmd;

	$RCmd = "	
					# Clean working environment. get rid of leftover
					# rm(list=ls())
					graphics.off()
					#setwd(\"$$OutPutDirRef\")
					png(filename=\"$$PNGFilePathRef\",width = 450, height = 300)
					# read source data
					datasource<-read.table(\"$$DataFilePathRef\",header=TRUE)
					attach(datasource)
					# par(las=3,ps=8)
					boxplot(value ~ group, data=datasource,col=rainbow(length(levels(group))))
					title(main=\"Test Time\", font.main=4, font.lab=1)
					# defect<-Frequency;
					# names(defect)<-ErrorCode;
					#pareto.chart(defect,col=rainbow(length(defect)));
					dev.off();
				";
	return $RCmd;

}

##################################################
#
####################################################
sub GenProcessCapabilityChart {
	my $op_textRef = shift;
	my $arrayoAref = shift;

	# my $datafilenameRef     = shift;
	# my $limitfilenameRef    = shift;
	my $itemlizeddataAoAref = shift;

	my $TitlesRef    = shift;
	my $SubTitlesRef = shift;
	my $UnitsRef     = shift;
	my $LowLimitsRef = shift;
	my $HiLimitsRef  = shift;

	my $SearchColListRef = shift;
	my $searchlistH_Ref  = shift;
	my $OutPutDirRef     = shift;
	my $WorkingDirRef    = shift;
	my $DecimalLenRef    = shift;
	my $FieldIndexHRef   = shift;

	my $i = 0;

	my @ProcessCapChartDataSetAoA;

	#foreach (@ResultTitles) {
	foreach (@$TitlesRef) {
		chomp();

		#my $subtitle = $ResultSubTitles[$i];
		my $subtitle = ${$SubTitlesRef}[$i];
		if ( !($subtitle) ) {
			$i++;
			next;
		}

		#$ResultLowLimit[$i] =~ s/^\s+|\s+$//g;
		${$LowLimitsRef}[$i] =~ s/^\s+|\s+$//g;

		#$ResultHiLimit[$i]  =~ s/^\s+|\s+$//g;
		${$HiLimitsRef}[$i] =~ s/^\s+|\s+$//g;

		my $lsl;

		#if (looks_like_number( $ResultLowLimit[$i] ) )    # &&
		if ( looks_like_number( ${$LowLimitsRef}[$i] ) ) {
			$lsl = ${$LowLimitsRef}[$i] * 10 / 10;
		}
		else {
			$lsl = 'NA';
		}

		my $usl;

		#if ( looks_like_number( $ResultHiLimit[$i] ) )    # &&
		if ( looks_like_number( ${$HiLimitsRef}[$i] ) ) {

			#$usl = $ResultHiLimit[$i] * 10 / 10;
			$usl = ${$HiLimitsRef}[$i] * 10 / 10;
		}
		else {
			$usl = 'NA';
		}

		my $target = 'NA';

		my $unit;

		#if ( $ResultUnits[$i] !~ /^$/i ) {
		if ( ${$UnitsRef}[$i] !~ /^$/i ) {

			#$unit = $ResultUnits[$i];
			$unit = ${$UnitsRef}[$i];
		}
		else {
			$unit = 'NA';
		}

		my $noofcommonfields = 11;

		#if ( @TestSearchList > $noofcommonfields ) {
		if ( @{$SearchColListRef} > $noofcommonfields ) {

			#if ( !defined $searchlistH{$_} ) {
			if ( !defined $$searchlistH_Ref{$_} ) {
				$i++;
				next;
			}

			#print "\$searchlistH{$_}=", $searchlistH{$_}, "\n";
		}

		# Generate boxplot chart for test time.
		if ( (/RFCAL|BFT/) && ( $subtitle =~ /Time/ ) ) {
			my $BoxplotDataFile =
			  $$OutPutDirRef . "/images/Testtime_Boxplot_datafile.txt";
			my $BoxPlotPNGfile =
			  $$OutPutDirRef . "/images/Testtime_Boxplot_PNGFile.png";
			sysopen( FH, $BoxplotDataFile, O_WRONLY | O_TRUNC | O_CREAT )
			  || die $!;
			print FH "group	\t	value  \n";
			foreach (@$arrayoAref) {
				$$_[ $$FieldIndexHRef{'STN_ID'} ] =~ s/^\s+|\s+$//g;
				print FH "$$_[$$FieldIndexHRef{'STN_ID'}] \t $$_[$i]  \n";
			}
			close(FH);

			GenRChart_Boxplot( $OutPutDirRef, \$BoxplotDataFile,
				\$BoxPlotPNGfile );

		#fcopy( $workingdir . '/DynamicWebPage/BoxPlotChart.png',$tmpfilename );
		# fcopy( $$WorkingDirRef . '/DynamicWebPage/BoxPlotChart.png',
		#	$tmpfilename );

			#$tmpfilename = $outputdir . '/images/TestTime-datasource.txt';
			#$tmpfilename = $$OutPutDirRef . '/images/TestTime-datasource.txt';

		  # fcopy( $outputdir . '/DataSource/boxplotdatafile.txt',$tmpfilename);
		  # fcopy( $$OutPutDirRef . '/DataSource/boxplotdatafile.txt',
		  #	$tmpfilename );
		}

		#
		s/^\s+|\s+$//g;
		s/\s+|-|\./_/g;
		$subtitle =~ s/^\s+|\s+$//g;
		$subtitle =~ s/\s+|\.|-/_/g;

		my $TestCaseName = $_ . '...' . $subtitle;

		# print "\n $i th: $TestCaseName \n";
		logmsg( $op_textRef, "$i th: $TestCaseName" );

		my @currentcolumn = ();
		@currentcolumn = map $_->[$i], @$arrayoAref;

		#print "current column before deleting non-numerical elements:".@currentcolumn, "\n";
		logmsg( $op_textRef, "Records before deleting non-numerical elements:" . @currentcolumn );

		foreach (@currentcolumn){
			# logmsg($op_textRef, $_);
		}
		# check if the records are number.
		# if not, the element will be elimated.
		#my $qtyrequirement = 3;
		my $qtyrequirement = 2;
		my $index          = 0;
		while ( $index <= $#currentcolumn ) {
			$currentcolumn[$index] =~ s/^\s+|\s+$//g;
			if ( !looks_like_number( $currentcolumn[$index] )
				|| not defined( $currentcolumn[$index] ) )
			{
				# remove element
				logmsg($op_textRef, "Remove element at $index");
				splice( @currentcolumn, $index, 1 );
				next;
			}
			else {
				$index++;
				next;
			}
		}

		if ( $#currentcolumn <= 3 ) {
			print "not engouth data to calculate indices ... \n";

			# build arrays for web page.
			push @$itemlizeddataAoAref,
			  [
				$i, $lsl, $usl, $unit, $TestCaseName,
				'No enough data to calculate statistical indices'
			  ];
			$i++;
			next;    # go to next column of data.
		}

		# Get rid of some extrem outliers.
		print "\@currentcolumn before percentile calculation: "
		  . @currentcolumn, "\n";
		my $stat = Statistics::Descriptive::Full->new();
		$stat->add_data( \@currentcolumn );
		my $Q1 = $stat->percentile(25);
		my $Q3 = $stat->percentile(75);
		my @emptyarray;
		$stat->add_data( \@emptyarray );

		my $IQR = $Q3 - $Q1;
		print "\$Q1=$Q1, \$Q3=$Q3, Q3-Q1= $IQR \n";
		print "extrem outlier border=", ( $Q1 - 3 * $IQR ), " , ",
		  ( $Q3 + 3 * $IQR ), "\n";

		#print "###total elements:" . @currentcolumn, "\n";
		logmsg( $op_textRef, "total elements:" . @currentcolumn );

		# make up a limit if both are 'NA'. otherwise, the qcc won't work.
		# if both lsl and usl are NA, assing one for lsl.
		if ( $lsl =~ /NA/i && $usl =~ /NA/i ) {
			$lsl = $Q1 - 1.5 * $IQR;
		}

		$index = 0;
		while ( $index <= $#currentcolumn ) {

			# Get rid of some extreme value
			# If the value is out of 6*$IQR and out of 3*Limit setting
			# it will be eliminated.
			#if (   $currentcolumn[$index] > ( $Q3 + 3 * $IQR )
			#	|| $currentcolumn[$index] < ( $Q1 - 3 * $IQR ) )
			my $CurrValue = $currentcolumn[$index];
			if (($CurrValue > ($Q3+3*$IQR) || $CurrValue < ($Q1-3*$IQR ))
				#&& ($CurrValue > $usl+abs($usl)*3 || $CurrValue < $lsl-abs($lsl)*3))
				&& ($CurrValue > $usl+abs($IQR)*3 || $CurrValue < $lsl-abs($IQR)*3))
			{
				#print "removing \$currentcolumn[$index]=$currentcolumn[$index] \n";
				logmsg( $op_textRef,
					"removing \$currentcolumn[$index]=$currentcolumn[$index]" );
				splice( @currentcolumn, $index, 1 );
				next;
			}
			else {
				$index++;
				next;
			}
		}

		#print "###elements in between:" . @currentcolumn, "\n";
		logmsg( $op_textRef, "Elements in between:" . @currentcolumn );

		# after the extrem outliers was eliminated,
		# the unique element qty need to be checked for statistics indices calculation.
		# check the unique element qty.
		my @uniquearray       = return_unique(@currentcolumn);
		my $sizeofuniquearray = @uniquearray;

		#if ( $sizeofuniquearray <= $qtyrequirement ) {
		if ( $sizeofuniquearray < $qtyrequirement ) {
			print "$sizeofuniquearray unique elements\n";
			print "not engouth data to calculate indices ... \n";
			print "@uniquearray \n";

			#build arrays for web page.
			push @$itemlizeddataAoAref,
			  [
				$i, $lsl, $usl, $unit, $TestCaseName,
				'No enough data to plot distribution'
			  ];
			$i++;
			next;    # go to next column of data.
		}

		# ready for chart generation
		# create limit file for process capability chart

		# make up a limit if both are 'NA'. otherwise, the qcc won't work.
		# if both lsl and usl are NA, assing one for lsl.
		#if ( $lsl =~ /NA/i && $usl =~ /NA/i ) {
		#	$lsl = $Q1 - 1.5 * $IQR;
		#}

	# sysopen( FH, $$limitfilenameRef, O_WRONLY | O_TRUNC | O_CREAT ) || die $!;
	# print FH "lsl \t t \t usl \t unit \n";
	# print FH "$lsl \t $target \t $usl \t $unit \n";
	# close(FH);
		########## output data ####################
		# define a array holds data ploatting chart needed.
		# [0] Column ID, [1] Limit file path [2] Data source file path
		my @IndividualDataSetA;

		# push the Column ID
		push( @IndividualDataSetA, $i );
		my $tmpfilenameLit =
		  $$OutPutDirRef . '/images/' . "$i-" . $TestCaseName . '-limits.txt';

		# fcopy( $$WorkingDirRef . '/DataSource/ProcessCapability-limits.txt',
		# $tmpfilenameLit );

		sysopen( FH, $tmpfilenameLit, O_WRONLY | O_TRUNC | O_CREAT ) || die $!;
		print FH "lsl \t t \t usl \t unit \n";
		print FH "$lsl \t $target \t $usl \t $unit \n";
		close(FH);

		# push the Limit file path
		push( @IndividualDataSetA, $tmpfilenameLit );

	 # write test result to test data file
	 # sysopen( FH, $$datafilenameRef, O_WRONLY | O_TRUNC | O_CREAT ) || die $!;
	 # print FH "$TestCaseName \n";
	 # foreach (@currentcolumn) {
	 # 	$_ =~ s/^\s+|\s+$//g;
	 # 	$_ = int( $_ * ( 10**$$DecimalLenRef ) ) / ( 10**$$DecimalLenRef );
	 # 	print FH "$_\n";
	 # }
	 # close(FH);

		my $tmpfilenameDatasource =
		    $$OutPutDirRef
		  . '/images/' . "$i-"
		  . $TestCaseName
		  . '-datasource.txt';

# fcopy( $$WorkingDirRef . '/DataSource/ProcessCapability.txt',	$tmpfilenameDatasource );

		# write test result to test data file
		sysopen( FH, $tmpfilenameDatasource, O_WRONLY | O_TRUNC | O_CREAT )
		  || die $!;
		print FH "$TestCaseName \n";
		foreach (@currentcolumn) {
			$_ =~ s/^\s+|\s+$//g;
			$_ = int( $_ * ( 10**$$DecimalLenRef ) ) / ( 10**$$DecimalLenRef );
			print FH "$_\n";
		}
		close(FH);

		push( @IndividualDataSetA, $tmpfilenameDatasource );

		# generate capability chart
		#my $ret = PerlRInterface( $WorkingDirRef, \"process_capability");
		#if ( $ret == 1 ) { print "Process capability chart was created. \n"; }
		#else             { print "failed to create Process capability chart"; }

		my $tmpfilenamePNG =
		  $$OutPutDirRef . '/images/' . "$i-" . $TestCaseName . '.png';

# print "tmpfilename for png file: $tmpfilenamePNG \n";
# fcopy( $$WorkingDirRef.'/DynamicWebPage/ProcessCapabilityChart.png', $tmpfilenamePNG );
# push the PNG file path
		push( @IndividualDataSetA, $tmpfilenamePNG );

		# push the current column data to AoA
		push( @ProcessCapChartDataSetAoA, \@IndividualDataSetA );

		#build arrays for web page.
		my @tmpAoA;
		push @$itemlizeddataAoAref,
		  [
			$i, $lsl, $usl, $unit, $TestCaseName,
			'./images/' . "$i-" . $TestCaseName . '.png'
		  ];

		# next test title field.
		$i++;
	}

	my $ret = GenRChart_ProcessCapability( $op_textRef, $WorkingDirRef,
		\"process_capability", \@ProcessCapChartDataSetAoA );
	if ( $ret == 1 ) {

		# print "Process capability chart was created. \n";
		logmsg( $op_textRef, "Process capability chart was created." );
		return 1;
	}
	else {

		#print "failed to create Process capability chart";
		logmsg( $op_textRef, "Failed to create Process capability chart" );
		return 0;
	}
}

#############################################################
sub GenHtmlFile {
	my $WorkingDireRef         = shift;
	my $buildsummaryAoAref     = shift;
	my $ECbreakdownAoAref      = shift;
	# my $ParetoChart_PNGFileRef = shift;
	my $Charts_PNGFileA_Ref	= shift;
	my $valueAoAref            = shift;
	my $OutPutFileRef          = shift;

	# some useful options (see below for full list)
	my $config = {

		#INCLUDE_PATH => './webpagetemplate',    # or list ref
		INCLUDE_PATH => $$WorkingDireRef . '/webpagetemplate',    # or list ref
		INTERPOLATE => 1,           # expand "$var" in plain text
		POST_CHOMP  => 1,           # cleanup whitespace
		PRE_PROCESS => 'header',    # prefix each template
		EVAL_PERL   => 1,           # evaluate Perl code blocks
	};

	# create Template object
	my $tt = Template->new($config);

	# define template variables for replacement
	my $vars = {
		records     => $valueAoAref,
		ECbreakdown	=> $ECbreakdownAoAref,
		#paretochart=> $$ParetoChart_PNGFileRef,
		charts		=> $Charts_PNGFileA_Ref,
		buildsummary=> $buildsummaryAoAref,

		#var3  	=> \@list,
		#var4  	=> \&code,
		#var5  	=> $object,
	};
	if ( defined($Charts_PNGFileA_Ref) ) {
		print "\$Charts_PNGFileA_Ref= join ",", @$Charts_PNGFileA_Ref. \n";
	}

	# specify input filename, or file handle, text reference, etc.
	# my $input = $dir."/webpagetemplate/example.tt";
	my $input = 'datasummary.tt';

	#my $output = './DynamicWebPage/main.html';
	#my $output = $outputdir.'/main.html';
	#my $output = $outputfile;
	my $output = $$OutPutFileRef;

	#print "html output file: $outputfile	\n";
	print "html output file: $$OutPutFileRef	\n";

	#$html_file_path=$output;

	#print "the html file path: $html_file_path. \n";
	# process input template, substituting variables
	$tt->process( $input, $vars, $output ) || die $tt->error();

	print "The processing is done!\n";
}

# ----------------------------------------------------------------------------
# &usage
# ----------------------------------------------------------------------------
sub logmsg {
	my $txt_ref = shift;
	my $logmsg  = shift;

	print "$logmsg \n";
	return 0 unless defined $txt_ref;

	$txt_ref->insert( 'end', "$logmsg \n" );
	$txt_ref->see('end');
}

# ----------------------------------------------------------------------------
# &usage
# ----------------------------------------------------------------------------
sub return_unique {
	my @array = @_;
	my %count;
	map { $count{$_} = 1 } @array;
	return sort keys(%count);
}

# ----------------------------------------------------------------------------
# &usage
# ----------------------------------------------------------------------------
sub count_unique {
	my @array = @_;
	my %count;
	map { $count{$_}++ } @array;

	#print them out:
	map { print "$_ = ${count{$_}}\n" } sort keys(%count);

	#or just return the hash:
	return %count;
}

1;
__END__
# Below is stub documentation for your module. You'd better edit it!

=head1 NAME

GetSummary - Perl extension for blah blah blah

=head1 SYNOPSIS

  use GetSummary;
  blah blah blah

=head1 DESCRIPTION

Stub documentation for GetSummary, created by h2xs. It looks like the
author of the extension was negligent enough to leave the stub
unedited.

Blah blah blah.

=head2 EXPORT

None by default.



=head1 SEE ALSO

Mention other useful documentation such as the documentation of
related modules or operating system documentation (such as man pages
in UNIX), or any relevant external documentation such as RFCs or
standards.

If you have a mailing list set up for your module, mention it here.

If you have a web site set up for your module, mention it here.

=head1 AUTHOR

A. U. Thor, E<lt>a.u.thor@a.galaxy.far.far.awayE<gt>

=head1 COPYRIGHT AND LICENSE

Copyright (C) 2011 by A. U. Thor

This library is free software; you can redistribute it and/or modify
it under the same terms as Perl itself, either Perl version 5.12.2 or,
at your option, any later version of Perl 5 you may have available.


=cut
